﻿import os
import re
import multiprocessing
import sys
import time
from datetime import datetime
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BASE_DIR)
# from Logs.log import log1
# from crawl.best_seller import seller_run
from crawl.unchange.JD import jd_review
from crawl.unchange.TMALL import tmall_review
from crawl.unchange.amazon import amazon_review
from crawl.unchange.bestbuy import bestbuy_review
# from crawl.unchange.JD import jd_review, new_jd
# from crawl.unchange.TMALL import tmall_review, new_tmall
# from crawl.unchange.amazon import amazon_review, new_amazon
# from crawl.unchange.bestbuy import bestbuy_review, new_bestbuy
# from crawl.unchange.biccamera import bicc_review, new_bicc
# from crawl.unchange.earphone import earphone_review, new_ear
# from crawl.unchange.kakaku import kakaku_review, new_kakaku
# from crawl.unchange.teufel import teufel_review, new_teufel
# from crawl.unchange.yodobashi import yodo_review, new_yodo
# from crawl.utils import log_info, get_urls, save as seller_run_save, close_db, select_count
# from crawl.utils import log_info, get_urls, save as seller_run_save, close_db, select_count


def list_split(urls):
    tms = []
    jds = []
    bys = []
    ams = []
    yos = []
    bis = []
    eas = []
    kas = []
    tes = []
    for u in urls:
        if u == None:
            continue
        if re.search("tmall\.com", u):
            tms.append(u)
        elif re.search("jd\.com", u):
            jds.append(u)
        elif re.search("\.bestbuy\.", u):
            bys.append(u)
        elif re.search("\.amazon\.", u):
            ams.append(u)
        elif re.search("\.yodobashi\.", u):
            yos.append(u)
        elif re.search("\.biccamera\.", u):
            bis.append(u)
        elif re.search("\.e-earphone\.", u):
            eas.append(u)
        elif re.search("kakaku\.", u):
            kas.append(u)
        elif re.search("\.teufelaudio\.", u):
            tes.append(u)
        else:
            # log_info("暂不支持此类网站数据抓取")
            print("暂不支持此类网站数据抓取")
    return tms, jds, bys, ams, yos, bis, eas, kas, tes


def run_new(urls):  # 新url调用接口
    tms, jds, bys, ams, yos, bis, eas, kas, tes = list_split(urls)
    process = []
    amazon = multiprocessing.Process(target=amazon_review.run, args=(ams,))
    bestbuy = multiprocessing.Process(target=bestbuy_review.run, args=(bys,))
    jd = multiprocessing.Process(target=jd_review.run, args=(jds,))
    tmall = multiprocessing.Process(target=tmall_review.run, args=(tms,))
    # yodo = multiprocessing.Process(target=yodo_review.run, args=(yos,))
    # bicc = multiprocessing.Process(target=bicc_review.run, args=(bis,))
    # ear = multiprocessing.Process(target=earphone_review.run, args=(eas,))
    # kakaku = multiprocessing.Process(target=kakaku_review.run, args=(kas,))
    # teufel = multiprocessing.Process(target=teufel_review.run, args=(tes,))
    process.append(amazon)
    # process.append(bestbuy)
    process.append(jd)
    process.append(tmall)
    # process.append(yodo)
    # process.append(bicc)
    # process.append(ear)
    # process.append(kakaku)
    # process.append(teufel)
    for p in process:
        p.start()
    for p in process:
        p.join()


# def run_today(urls):  # 抓取每天新评论接口
#     tms, jds, bys, ams, yos, bis, eas, kas, tes = list_split(urls)
#     process = []
#     amazon = multiprocessing.Process(target=new_amazon.main, args=(ams,))
#     bestbuy = multiprocessing.Process(target=new_bestbuy.main, args=(bys,))
#     jd = multiprocessing.Process(target=new_jd.main, args=(jds,))
#     tmall = multiprocessing.Process(target=new_tmall.main, args=(tms,))
#     yodo = multiprocessing.Process(target=new_yodo.main, args=(yos,))
#     bicc = multiprocessing.Process(target=new_bicc.main, args=(bis,))
#     ear = multiprocessing.Process(target=new_ear.main, args=(eas,))
#     kakaku = multiprocessing.Process(target=new_kakaku.main, args=(kas,))
#     teufel = multiprocessing.Process(target=new_teufel.main, args=(tes,))
#     process.append(amazon)
#     process.append(bestbuy)
#     process.append(jd)
#     process.append(tmall)
#     process.append(yodo)
#     process.append(bicc)
#     process.append(ear)
#     process.append(kakaku)
#     process.append(teufel)
#     for th in process:
#         th.start()
#     for th in process:
#         th.join()


def main():
    # results = get_urls()
    results = [("https://item.jd.com/65398375780.html", "jd0"), ("https://detail.tmall.com/item.htm?spm=a211oj.15203896.2940888250.14&id=591771447883", "TM0"), ("https://www.amazon.com/-/zh/dp/B078N2KZDF/ref=sr_1_13", "am0")]
    list1 = []
    list2 = []
    for result in results:
        url = result[0]
        sku_detail_id = result[1]
        # count = select_count(sku_detail_id)
        count = 0
        if count == 0:
            list1.append(url+"$$$"+sku_detail_id)
        else:
            list2.append(url+"$$$"+sku_detail_id)
    if list1:
        run_new(list1)
    # if list2:
    #     run_today(list2)
    # close_db()
    print("抓取End!")
    # log1.info("抓取End!")


def running():
    # seller_run.run()
    main()


# 运行入口
if __name__ == '__main__':
   running()
